from flask import request,Blueprint,jsonify
import requests
import re
from bs4 import BeautifulSoup
def doCrawl(sourceType):
    from backstage.data import DataType
    list=[]
    if sourceType==DataType.XLWB.name:
        list=crawlXLWB()
    elif sourceType==DataType.JRTT.name:
        print(DataType.JRTT.name)
    elif sourceType==DataType.SGSS.name:
        print(DataType.SGSS.name)
    else :
        print("暂不支持的类型")
    # r = reque/sts.get(url, timeout=timeout)
    import backstage.db as DB
    for item in list:
        sql="insert into sys_data_source(content,url,sourceType) values('{content}','{url}','{sourceType}')".format(
            content=item.get("content"),url=item.get("url"),sourceType=sourceType
        )
        DB.controlDB(sql)
    return

def crawlXLWB():
    baseUrl="https://s.weibo.com/weibo?q={q}&wvr=6&b=1&Refer={Refer}".format(q="校园疫情",Refer="SWeibo_box")

    r = requests.get(baseUrl, timeout=3000)
    r.raise_for_status()
    r.encoding = "utf-8"
    html=r.text
    soup = BeautifulSoup(html, 'html.parser')
    # node - type = "feed_list_content"
    items=soup.find_all('p',attrs={'node-type':'feed_list_content','class':'txt'})
    list=[]
    for item in items:
        aTag=item.select('a')
        for tag in aTag:
            # obj = re.findall(re.compile(r'<a href="(.*?)" target="_blank">(.*)</a>'), tag)
            url=tag['href']
            content=tag.string
            if content is not None:
                list.append({
                "url":url,
                "content":content
                })
    return list






import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
from wordcloud import WordCloud, ImageColorGenerator
import jieba
def GetWordCloud(str):
   img = Image.open(r"../static/images/back.jpg")
   background_image = np.array(img)
   cut_text = " ".join(jieba.cut(str))
   wordcloud = WordCloud(
       # 设置字体，不然会出现口字乱码，文字的路径是电脑的字体一般路径，可以换成别的
       font_path="C:/Windows/Fonts/STSONG.TTF",
       background_color="white",
       # mask参数=图片背景，必须要写上，另外有mask参数再设定宽高是无效的
       mask=background_image).generate(cut_text)
   # 生成颜色值
   image_colors = ImageColorGenerator(background_image)
   # 下面代码表示显示图片
   plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation="bilinear")
   plt.axis("off")
   # plt.show()
   plt.savefig("../static/images/WordCloud.jpg")


import shutil
import jieba
import pdb
from sklearn.feature_extraction.text import TfidfVectorizer
def tf(wei):
    vector = TfidfVectorizer(stop_words=['的','啊'])#引用停用词
    tfidf = vector.fit_transform(wei)#
    d = tfidf.toarray()
    word=vector.get_feature_names()
    return d

#K-means 算法聚类
from sklearn.cluster import KMeans as KM
def km(d):
    k = 4 #将所有文本分成4类
    clf = KM(k)
    kmns = clf.fit_predict(d)
    return kmns

wei=["张三去了北京","西安","北京大胡同","太原在山西"]
tfs = tf(wei)  #将文本向量化，TF-IDF和标准化
kms = km(tfs)    #K--means聚类算法聚类

print()

